#!/bin/bash
stop_file="stop_file"
[ -f "${stop_file}" ] && exit
ha_name="${1}"
mysql_user="dbms_monitor_r"
mysql_passwd="4bb51ad0-8964-11e8-b8fe-cf6796b9df33"
md5_res="md5_res"
mark_file=".mark"
log_dir="log"
ha_list_file="${log_dir}/ha.log"
send_log="${log_dir}/send.log"
monitor_log="${log_dir}/monitor.log"
node_state=0
[ ! -d "${log_dir}" ] && mkdir ${log_dir}
[ ! -d "${md5_res}" ] && mkdir ${md5_res}
product_info="$(grep "^${ha_name}:" ${ha_list_file}|grep ":写节点:"|awk -F: '{print $5}'|tail -1)"
project_info="$(grep "^${ha_name}:" ${ha_list_file}|grep ":写节点:"|awk -F: '{print $6}'|tr "\n" ","|sed 's/,$//g')"
write_node=$(grep "^${ha_name}:" ${ha_list_file}|grep ":写节点:"|awk -F: '{print $3}'|tail -1)
node_port=$(grep "^${ha_name}:" ${ha_list_file}|awk -F: '{print $4}'|tail -1)

write_mysql_comm="mysql -u${mysql_user} -p${mysql_passwd} -h${write_node} -P${node_port}"
read_node_array=($(grep "^${ha_name}:" ${ha_list_file}|grep -v ":写节点:"|awk -F: '{print $3}'|tr " " "\n"|sort|uniq|tr "\n" " "))

function f_send_monitor()
{
    error_node=(${1})
    master_node=(${2})
    send_file="${3}"
    state="${4}"
    node_role="${5}"
    node_gtid=(${6})
    write_gtid=(${7})
    mgr_status="${8}"
    if [ "${state}x" == "OKx" ]
    then
        show_state="已恢复"
        sts=2
        gtid_behind=0
    else
        show_state="故障中"
        sts=0
        write_gtid_count=$(awk -F- '{print "0"$NF}' <<< "${write_gtid}"|sed 's/[^0-9]//g'|tail -1)
        node_gtid_count=$(awk -F- '{print "0"$NF}' <<< "${node_gtid}"|sed 's/[^0-9]//g'|tail -1)
        gtid_behind=$(($(awk '{print $0-0}' <<< "${write_gtid_count}")-$(awk '{print $0-0}' <<< "${node_gtid_count}")))
    fi
    t_time="$(date "+%F %H:%M:%S")"
cat <<EOF > ${send_file}
告警通知
告警状态   : ${show_state}
故障节点   : ${error_node[0]}:${error_node[1]}
节点角色   : ${node_role}
写节点ip   : ${master_node[0]}:${master_node[1]}
NODEGTID   : ${node_gtid}
WRITEGTID  : ${write_gtid}
BehindGTID : ${gtid_behind}
告警时间   : ${t_time}
告警级别   : 严重告警
产品线名   : ${product_info}
项目名称   : ${project_info}
高可用组   : ${ha_name}

--以下四个值对应sys视图的四个状态值--
viable_candidate:${mgr_status[0]}
read_only:${mgr_status[1]}
transactions_behind:${mgr_status[2]}
transactions_to_cert:${mgr_status[3]}
EOF
    cat <<EOF |mysql -umoshan -p'Actionsky0821!@' -h192.168.2.142 -P3306 db_monitor
insert into t_monitor_alert_sendlist(type,title,content,usergroup_id,sts,create_time,send_time) select 'mysql','DB Monitor:MGR延迟告警(${error_node[0]}:${error_node[1]})-${show_state}','$(cat ${send_file})',1,${sts},'${t_time}','${t_time}'
EOF
    python3 send.py ${send_file} "DB Monitor:MGR延迟告警(${error_node[0]}:${error_node[1]})-${show_state}" >/dev/null 
    if [ $? -eq 0 ]
    then
       send_state="OK"
    else
       send_state="ERROR"
    fi
    echo "insert into t_monitor_alert_sendlist(type,title,content,usergroup_id,sts,create_time,send_time) select 'mysql','MGR status alarm [${error_node[0]}:${error_node[1]}]','$(cat ${send_file})',1,0,'${t_time}','${t_time}'" >> ${monitor_log}
    if [ "${state}x" == "OKx" ]
    then
        echo -e "\033[32m[${t_time}] [MGR STATUS ${state}] [SEND STATUS ${send_state}] ${error_node[0]}:${error_node[1]} ${product_info} ${project_info} ${ha_name} \033[0m" >> ${send_log}
        rm -f ${send_file}
    else
        echo -e "\033[31m[${t_time}] [MGR STATUS ${state}] [SEND STATUS ${send_state}] ${error_node[0]}:${error_node[1]} ${product_info} ${project_info} ${ha_name} \033[0m" >> ${send_log}
    fi
}


for ((i=0;i<${#read_node_array[@]};i++))
do
{
    node_res_file="${md5_res}/${read_node_array[${i}]}_${node_port}.log"
    write_res_file="${md5_res}/write_${read_node_array[${i}]}_${node_port}.log"
    > ${node_res_file}
    > ${write_res_file}
    node_mysql_comm="mysql -u${mysql_user} -p${mysql_passwd} -h${read_node_array[${i}]} -P${node_port}"
    for ((j=0;j<10;j++))
    do
        ${node_mysql_comm} -e "show master status\G" 2>/dev/null|grep -A 1000 "Executed_Gtid_Set:"|sed 's/Executed_Gtid_Set: //'|tr -d ","|sort|md5sum |awk '{print $1}' >> ${node_res_file} &
        ${write_mysql_comm} -e "show master status\G" 2>/dev/null|grep -A 1000 "Executed_Gtid_Set:"|sed 's/Executed_Gtid_Set: //'|tr -d ","|sort|md5sum |awk '{print $1}' >> ${write_res_file}
        sleep 1
    done
}&
done
wait

for ((i=0;i<${#read_node_array[@]};i++))
do
    node_role="$(grep ${read_node_array[${i}]} ${ha_list_file}|grep "^${ha_name}:"|grep ${node_port}|awk -F: '{print $2}'|tail -1)"
    node_res_file="${md5_res}/${read_node_array[${i}]}_${node_port}.log"
    [ "$(wc -l < ${node_res_file} 2>/dev/null)x" == "0x" ] && continue
    node_send_file="${md5_res}/${read_node_array[${i}]}_${node_port}.send"
    write_res_file="${md5_res}/write_${read_node_array[${i}]}_${node_port}.log"
    if [ "$(sort < ${node_res_file} 2>/dev/null|uniq|wc -l)x" == "1x" ]
    then
        if [ "$(sort < ${write_res_file} 2>/dev/null|uniq|wc -l)x" == "1x" ]
        then
            node_state=1
        else
            node_mysql_comm="mysql -u${mysql_user} -p${mysql_passwd} -h${read_node_array[${i}]} -P${node_port}"
            md5_res_var="$(${node_mysql_comm} -e "show master status\G" 2>/dev/null|grep -A 1000 "Executed_Gtid_Set:"|sed 's/Executed_Gtid_Set: //'|tr -d ","|sort|md5sum |awk '{print $1}')"
            if [ "$(grep -c ${md5_res_var} ${node_res_file} 2>/dev/null)x" == "0x" ]
            then
                node_state=1
            else
                node_state=0
            fi
        fi
    else
        node_state=1
    fi
    if [ "${node_state}x" == "1x" ]
    then
        if [ -f "${node_send_file}" ]
        then
            rm -f ${node_send_file} && f_send_monitor "${read_node_array[${i}]} ${node_port}" "${write_node} ${node_port}" "${node_send_file}" "OK" "${node_role}"
        fi
    elif [ "${node_state}x" == "0x" ]
    then
        gtid_group=$(${write_mysql_comm} -NBe "show variables like 'group_replication_group_name'"|awk '{print $2}')
        node_gtid_var=($(${node_mysql_comm} -e "show master status\G" 2>/dev/null|grep -A 1000 "Executed_Gtid_Set:"|sed 's/Executed_Gtid_Set: //'|tr -d ","|sort|grep "${gtid_group}"))
        [ "${node_gtid_var}x" == "x" ] && node_gtid_var="该节点访问异常" 
        write_gtid_var=($(${write_mysql_comm} -e "show master status\G" 2>/dev/null|grep -A 1000 "Executed_Gtid_Set:"|sed 's/Executed_Gtid_Set: //'|tr -d ","|sort|grep "${gtid_group}"))
        [ "${write_gtid_var}x" == "x" ] && write_gtid_var="写节点访问异常" 
        if [ "${node_gtid_var}x" == "${write_gtid_var}x" ]
        then
            if [ -f "${node_send_file}" ]
            then
                rm -f ${node_send_file}
            fi
        else
            mgr_status=($(${node_mysql_comm} -NBe "select * from sys.gr_member_routing_candidate_status"))
            f_send_monitor "${read_node_array[${i}]} ${node_port}" "${write_node} ${node_port}" "${node_send_file}" "ERROR" "${node_role}" "${node_gtid_var[@]}" "${write_gtid_var[@]}" "${mgr_status[@]}"
        fi
    fi
done
[ -f "${mark_file}" ] && rm -f ${mark_file}